agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v20260912 4/4] Make printtup a bit faster (intermediate state). 330+ messages / 2 participants [nested] [flat]
* [PATCH v20260912 4/4] Make printtup a bit faster (intermediate state). @ 2024-09-12 10:03 Andy Fan <zhihuifan1213@163.com> 0 siblings, 0 replies; 330+ messages in thread From: Andy Fan @ 2024-09-12 10:03 UTC (permalink / raw) Currently the out function usually allocate its own memory and fill it with the cstring. After the printtup get the cstring, printtup computes it string length and copy it to its own StringInfo. So there are some wastage in this workflow. In the desired case, out function should take a StringInfo as a input and fill the data to StringInfo's buffer directly. Within this way, there is no extra memory allocate, memory copy and probably avoid the most strlen since the most of the outfunction can compute it easily. for example a). snprintf return the length encoded string, b). the varlena's header has a strlen. c). we know the start position before we encode a Datum and we know the end position after the Datum encoding, so the length would be similar as 'end_pos - start_pos'. Since we have 79 out functions to change, this patch just finish part of them by using a new print function and wish a review of it. If there are anything wrong, it is better know them earlier. --- src/backend/access/common/printtup.c | 80 ++++++++++++++++++-- src/backend/utils/adt/char.c | 32 ++++++++ src/backend/utils/adt/date.c | 74 ++++++++++++++++++- src/backend/utils/adt/datetime.c | 17 ++++- src/backend/utils/adt/float.c | 53 +++++++++++++- src/backend/utils/adt/int.c | 32 ++++++++ src/backend/utils/adt/int8.c | 16 ++++ src/backend/utils/adt/numeric.c | 68 +++++++++++++++-- src/backend/utils/adt/oid.c | 16 ++++ src/backend/utils/adt/timestamp.c | 106 ++++++++++++++++++++++++++- src/backend/utils/adt/varchar.c | 25 +++++++ src/backend/utils/adt/varlena.c | 16 ++++ src/include/catalog/pg_proc.dat | 83 ++++++++++++++++++++- src/include/lib/stringinfo.h | 19 +++++ src/include/utils/date.h | 2 +- src/include/utils/datetime.h | 8 +- 16 files changed, 618 insertions(+), 29 deletions(-) diff --git a/src/backend/access/common/printtup.c b/src/backend/access/common/printtup.c index 616bdafd395..860e67cfcc9 100644 --- a/src/backend/access/common/printtup.c +++ b/src/backend/access/common/printtup.c @@ -19,6 +19,7 @@ #include "libpq/pqformat.h" #include "libpq/protocol.h" #include "tcop/pquery.h" +#include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/memdebug.h" #include "utils/memutils.h" @@ -50,6 +51,7 @@ typedef struct bool typisvarlena; /* is it varlena (ie possibly toastable)? */ int16 format; /* format code for this column */ FmgrInfo finfo; /* Precomputed call info for output fn */ + FmgrInfo p_finfo; /* Precomputed call info for print fn if any */ } PrinttupAttrInfo; typedef struct @@ -244,6 +246,47 @@ SendRowDescriptionMessage(StringInfo buf, TupleDesc typeinfo, pq_endmessage_reuse(buf); } +static Oid +get_type_printfn_tmp(Oid type) +{ + switch(type) + { + case OIDOID: + return F_OIDPRINT; + case TEXTOID: + return F_TEXTPRINT; + case FLOAT4OID: + return F_FLOAT4PRINT; + case FLOAT8OID: + return F_FLOAT8PRINT; + case INT2OID: + return F_INT2PRINT; + case INT4OID: + return F_INT4PRINT; + case INT8OID: + return F_INT8PRINT; + case TIMEOID: + return F_TIMEPRINT; + case TIMETZOID: + return F_TIMETZPRINT; + case TIMESTAMPOID: + return F_TIMESTAMPPRINT; + case TIMESTAMPTZOID: + return F_TIMESTAMPTZPRINT; + case INTERVALOID: + return F_INTERVAL_PRINT; + case NUMERICOID: + return F_NUMERIC_PRINT; + case BPCHAROID: + return F_BPCHARPRINT; + case VARCHAROID: + return F_VARCHARPRINT; + case CHAROID: + return F_CHARPRINT; + } + return InvalidOid; +} + /* * Get the lookup info that printtup() needs */ @@ -275,10 +318,18 @@ printtup_prepare_info(DR_printtup *myState, TupleDesc typeinfo, int numAttrs) thisState->format = format; if (format == 0) { - getTypeOutputInfo(attr->atttypid, - &thisState->typoutput, - &thisState->typisvarlena); - fmgr_info(thisState->typoutput, &thisState->finfo); + Oid print_fn = get_type_printfn_tmp(attr->atttypid); + if (print_fn != InvalidOid) + fmgr_info(print_fn, &thisState->p_finfo); + else + { + getTypeOutputInfo(attr->atttypid, + &thisState->typoutput, + &thisState->typisvarlena); + fmgr_info(thisState->typoutput, &thisState->finfo); + /* mark print function is invalid */ + thisState->p_finfo.fn_oid = InvalidOid; + } } else if (format == 1) { @@ -356,10 +407,23 @@ printtup(TupleTableSlot *slot, DestReceiver *self) if (thisState->format == 0) { /* Text output */ - char *outputstr; - - outputstr = OutputFunctionCall(&thisState->finfo, attr); - pq_sendcountedtext(buf, outputstr, strlen(outputstr)); + if (thisState->p_finfo.fn_oid) + { + /* + * Use print function if it is defined. + * + * XXX: we can remove this if statement once we refactor all + * the out function. + */ + FunctionCall2(&thisState->p_finfo, attr, PointerGetDatum(buf)); + } + else + { + char *outputstr; + + outputstr = OutputFunctionCall(&thisState->finfo, attr); + pq_sendcountedtext(buf, outputstr, strlen(outputstr)); + } } else { diff --git a/src/backend/utils/adt/char.c b/src/backend/utils/adt/char.c index 698863924ee..6d1d9403c2d 100644 --- a/src/backend/utils/adt/char.c +++ b/src/backend/utils/adt/char.c @@ -83,6 +83,38 @@ charout(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +charprint(PG_FUNCTION_ARGS) +{ + char ch = PG_GETARG_CHAR(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + char *result; + uint32 data_len; + + result = outStringReserveLen(buf, 5); + + if (IS_HIGHBIT_SET(ch)) + { + result[0] = '\\'; + result[1] = TOOCTAL(((unsigned char) ch) >> 6); + result[2] = TOOCTAL((((unsigned char) ch) >> 3) & 07); + result[3] = TOOCTAL(((unsigned char) ch) & 07); + result[4] = '\0'; + data_len = 4; + } + else + { + /* This produces acceptable results for 0x00 as well */ + result[0] = ch; + result[1] = '\0'; + data_len = 1; + } + + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * charrecv - converts external binary format to char * diff --git a/src/backend/utils/adt/date.c b/src/backend/utils/adt/date.c index c3327440380..4d606e7888f 100644 --- a/src/backend/utils/adt/date.c +++ b/src/backend/utils/adt/date.c @@ -196,6 +196,31 @@ date_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +date_print(PG_FUNCTION_ARGS) +{ + DateADT date = PG_GETARG_DATEADT(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + char *data; + uint32 data_len; + + struct pg_tm tt, + *tm = &tt; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + + if (DATE_NOT_FINITE(date)) + data_len = EncodeSpecialDate(date, data); + else + { + j2date(date + POSTGRES_EPOCH_JDATE, + &(tm->tm_year), &(tm->tm_mon), &(tm->tm_mday)); + data_len = EncodeDateOnly(tm, DateStyle, data); + } + outStringCompletePhase(buf, data_len); + PG_RETURN_VOID(); +} + /* * date_recv - converts external binary format to date */ @@ -291,13 +316,21 @@ make_date(PG_FUNCTION_ARGS) /* * Convert reserved date values to string. */ -void +int EncodeSpecialDate(DateADT dt, char *str) { if (DATE_IS_NOBEGIN(dt)) + { strcpy(str, EARLY); + /* the return value can be computed at compiling time. */ + return strlen(EARLY); + } else if (DATE_IS_NOEND(dt)) + { strcpy(str, LATE); + /* the return value can be computed at compiling time. */ + return strlen(LATE); + } else /* shouldn't happen */ elog(ERROR, "invalid argument for EncodeSpecialDate"); } @@ -1603,6 +1636,25 @@ time_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +time_print(PG_FUNCTION_ARGS) +{ + TimeADT time = PG_GETARG_TIMEADT(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + time2tm(time, tm, &fsec); + data_len = EncodeTimeOnly(tm, fsec, false, 0, DateStyle, data); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * time_recv - converts external binary format to time */ @@ -2417,6 +2469,26 @@ timetz_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +timetz_print(PG_FUNCTION_ARGS) +{ + TimeTzADT *time = PG_GETARG_TIMETZADT_P(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + char *data; + uint32 data_len; + int tz; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + timetz2tm(time, tm, &fsec, &tz); + data_len = EncodeTimeOnly(tm, fsec, true, tz, DateStyle, data); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * timetz_recv - converts external binary format to timetz */ diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index 83c3c85305b..85da5780870 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -4346,9 +4346,10 @@ EncodeTimezone(char *str, int tz, int style) /* EncodeDateOnly() * Encode date as local time. */ -void +int EncodeDateOnly(struct pg_tm *tm, int style, char *str) { + char *start = str; Assert(tm->tm_mon >= 1 && tm->tm_mon <= MONTHS_PER_YEAR); switch (style) @@ -4420,6 +4421,7 @@ EncodeDateOnly(struct pg_tm *tm, int style, char *str) str += 3; } *str = '\0'; + return str - start; } @@ -4430,10 +4432,13 @@ EncodeDateOnly(struct pg_tm *tm, int style, char *str) * a time zone (the difference between time and timetz types), tz is the * numeric time zone offset, style is the date style, str is where to write the * output. + * + * returns the strlen of the encoded format. */ -void +int EncodeTimeOnly(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, int style, char *str) { + char *start = str; str = pg_ultostr_zeropad(str, tm->tm_hour, 2); *str++ = ':'; str = pg_ultostr_zeropad(str, tm->tm_min, 2); @@ -4442,6 +4447,7 @@ EncodeTimeOnly(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, int style, if (print_tz) str = EncodeTimezone(str, tz, style); *str = '\0'; + return str - start; } @@ -4460,11 +4466,14 @@ EncodeTimeOnly(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, int style, * ISO - yyyy-mm-dd hh:mm:ss+/-tz * German - dd.mm.yyyy hh:mm:ss tz * XSD - yyyy-mm-ddThh:mm:ss.ss+/-tz + * + * return the strlen of the encoded data. */ -void +int EncodeDateTime(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, const char *tzn, int style, char *str) { int day; + char *start = str; Assert(tm->tm_mon >= 1 && tm->tm_mon <= MONTHS_PER_YEAR); @@ -4624,6 +4633,8 @@ EncodeDateTime(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, const char str += 3; } *str = '\0'; + + return str - start; } diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c index 13ceade129e..76ad00c60d9 100644 --- a/src/backend/utils/adt/float.c +++ b/src/backend/utils/adt/float.c @@ -373,6 +373,32 @@ float4out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(ascii); } + +Datum +float4print(PG_FUNCTION_ARGS) +{ + float4 num = PG_GETARG_FLOAT4(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + int data_len; + char *ascii; + int ndig = FLT_DIG + extra_float_digits; + + ascii = outStringReserveLen(buf, 32); + + if (extra_float_digits > 0) + data_len = float_to_shortest_decimal_buf(num, ascii); + else + data_len = pg_strfromd(ascii, 32, ndig, num); + if (data_len == -1) + { + /* XXX, think more of this. */ + elog(ERROR, "failed on float4print"); + } + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * float4recv - converts external binary format to float4 */ @@ -563,10 +589,35 @@ Datum float8out(PG_FUNCTION_ARGS) { float8 num = PG_GETARG_FLOAT8(0); + int len; - PG_RETURN_CSTRING(float8out_internal(num)); + PG_RETURN_CSTRING(float8out_internal(num, NULL, &len)); } +Datum +float8print(PG_FUNCTION_ARGS) +{ + float8 num = PG_GETARG_FLOAT8(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + int data_len; + char *ascii; + + ascii = outStringReserveLen(buf, 32); + + float8out_internal(num, ascii, &data_len); + + if (data_len == -1) + { + /* XXX, think more of this. */ + elog(ERROR, "failed on float8print"); + } + + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + + /* * float8out_internal - guts of float8out() * diff --git a/src/backend/utils/adt/int.c b/src/backend/utils/adt/int.c index 4c894a49d5d..5a121a46b94 100644 --- a/src/backend/utils/adt/int.c +++ b/src/backend/utils/adt/int.c @@ -80,6 +80,22 @@ int2out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } + +Datum +int2print(PG_FUNCTION_ARGS) +{ + int16 arg1 = PG_GETARG_INT16(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, 7); + data_len = pg_itoa(arg1, data); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * int2recv - converts external binary format to int2 */ @@ -333,6 +349,22 @@ int4out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +int4print(PG_FUNCTION_ARGS) +{ + int32 arg1 = PG_GETARG_INT32(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, 12); + data_len = pg_ltoa(arg1, data); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + + /* * int4recv - converts external binary format to int4 */ diff --git a/src/backend/utils/adt/int8.c b/src/backend/utils/adt/int8.c index 19bb30f2d0f..8580c273792 100644 --- a/src/backend/utils/adt/int8.c +++ b/src/backend/utils/adt/int8.c @@ -76,6 +76,22 @@ int8out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +int8print(PG_FUNCTION_ARGS) +{ + int64 arg1 = PG_GETARG_INT64(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, MAXINT8LEN + 1); + data_len = pg_lltoa(arg1, data); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + + /* * int8recv - converts external binary format to int8 */ diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index cb23dfe9b95..84719a79ae8 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -509,7 +509,7 @@ static bool set_var_from_non_decimal_integer_str(const char *str, static void set_var_from_num(Numeric num, NumericVar *dest); static void init_var_from_num(Numeric num, NumericVar *dest); static void set_var_from_var(const NumericVar *value, NumericVar *dest); -static char *get_str_from_var(const NumericVar *var); +static char *get_str_from_var(const NumericVar *var, StringInfo buf); static char *get_str_from_var_sci(const NumericVar *var, int rscale); static void numericvar_serialize(StringInfo buf, const NumericVar *var); @@ -820,11 +820,52 @@ numeric_out(PG_FUNCTION_ARGS) */ init_var_from_num(num, &x); - str = get_str_from_var(&x); + str = get_str_from_var(&x, NULL); PG_RETURN_CSTRING(str); } +Datum +numeric_print(PG_FUNCTION_ARGS) +{ + Numeric num = PG_GETARG_NUMERIC(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + + NumericVar x; + + /* + * Handle NaN and infinities + */ + if (NUMERIC_IS_SPECIAL(num)) + { + const char* special_str; + char *data; + uint32 data_len; + + if (NUMERIC_IS_PINF(num)) + special_str = "Infinity"; + else if (NUMERIC_IS_NINF(num)) + special_str = "-Infinity"; + else + special_str = "NaN"; + + data_len = strlen(special_str) + 1; + data = outStringReserveLen(buf, data_len); + memcpy(data, special_str, data_len); + outStringCompletePhase(buf, data_len); + PG_RETURN_VOID(); + } + + /* + * Get the number in the variable format. + */ + init_var_from_num(num, &x); + + (void) get_str_from_var(&x, buf); + + PG_RETURN_VOID(); +} + /* * numeric_is_nan() - * @@ -1027,7 +1068,7 @@ numeric_normalize(Numeric num) init_var_from_num(num, &x); - str = get_str_from_var(&x); + str = get_str_from_var(&x, NULL); /* If there's no decimal point, there's certainly nothing to remove. */ if (strchr(str, '.') != NULL) @@ -7251,7 +7292,7 @@ set_var_from_var(const NumericVar *value, NumericVar *dest) * Returns a palloc'd string. */ static char * -get_str_from_var(const NumericVar *var) +get_str_from_var(const NumericVar *var, StringInfo buf) { int dscale; char *str; @@ -7279,7 +7320,14 @@ get_str_from_var(const NumericVar *var) if (i <= 0) i = 1; - str = palloc(i + dscale + DEC_DIGITS + 2); + if (buf == NULL) + { + str = palloc(i + dscale + DEC_DIGITS + 2); + } + else + { + str = outStringReserveLen(buf, i + dscale + DEC_DIGITS + 2); + } cp = str; /* @@ -7378,6 +7426,12 @@ get_str_from_var(const NumericVar *var) * terminate the string and return it */ *cp = '\0'; + + if (buf != NULL) + { + uint32 data_len = cp - str; + outStringCompletePhase(buf, data_len); + } return str; } @@ -7451,7 +7505,7 @@ get_str_from_var_sci(const NumericVar *var, int rscale) power_ten_int(exponent, &tmp_var); div_var(var, &tmp_var, &tmp_var, rscale, true, true); - sig_out = get_str_from_var(&tmp_var); + sig_out = get_str_from_var(&tmp_var, NULL); free_var(&tmp_var); @@ -8004,7 +8058,7 @@ numericvar_to_double_no_overflow(const NumericVar *var) double val; char *endptr; - tmp = get_str_from_var(var); + tmp = get_str_from_var(var, NULL); /* unlike float8in, we ignore ERANGE from strtod */ val = strtod(tmp, &endptr); diff --git a/src/backend/utils/adt/oid.c b/src/backend/utils/adt/oid.c index a3419728971..96df114eaf6 100644 --- a/src/backend/utils/adt/oid.c +++ b/src/backend/utils/adt/oid.c @@ -53,6 +53,22 @@ oidout(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +oidprint(PG_FUNCTION_ARGS) +{ + Oid o = PG_GETARG_OID(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + uint32 data_len; + char *data; + + /* 12 is the max length for an oid's text presentation. */ + data = outStringReserveLen(buf, 12); + data_len = pg_snprintf(data, 12, "%u", o); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * oidrecv - converts external binary format to oid */ diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index 288d696be77..27b546deb74 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -87,7 +87,7 @@ static bool AdjustIntervalForTypmod(Interval *interval, int32 typmod, static TimestampTz timestamp2timestamptz(Timestamp timestamp); static Timestamp timestamptz2timestamp(TimestampTz timestamp); -static void EncodeSpecialInterval(const Interval *interval, char *str); +static int EncodeSpecialInterval(const Interval *interval, char *str); static void interval_um_internal(const Interval *interval, Interval *result); /* common code for timestamptypmodin and timestamptztypmodin */ @@ -244,6 +244,33 @@ timestamp_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +timestamp_print(PG_FUNCTION_ARGS) +{ + Timestamp timestamp = PG_GETARG_TIMESTAMP(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + + if (TIMESTAMP_NOT_FINITE(timestamp)) + data_len = EncodeSpecialTimestamp(timestamp, data); + else if (timestamp2tm(timestamp, NULL, tm, &fsec, NULL, NULL) == 0) + data_len = EncodeDateTime(tm, fsec, false, 0, NULL, DateStyle, data); + else + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * timestamp_recv - converts external binary format to timestamp */ @@ -789,6 +816,36 @@ timestamptz_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +timestamptz_print(PG_FUNCTION_ARGS) +{ + TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + int tz; + const char *tzn; + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + + if (TIMESTAMP_NOT_FINITE(timestamp)) + data_len = EncodeSpecialTimestamp(timestamp, data); + else if (timestamp2tm(timestamp, &tz, tm, &fsec, &tzn, NULL) == 0) + data_len = EncodeDateTime(tm, fsec, true, tz, tzn, DateStyle, data); + else + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + + /* * timestamptz_recv - converts external binary format to timestamptz */ @@ -983,6 +1040,35 @@ interval_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(result); } +Datum +interval_print(PG_FUNCTION_ARGS) +{ + Interval *span = PG_GETARG_INTERVAL_P(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + struct pg_itm tt, + *itm = &tt; + char *data; + uint32 data_len; + + data = outStringReserveLen(buf, MAXDATELEN + 1); + + if (INTERVAL_NOT_FINITE(span)) + data_len = EncodeSpecialInterval(span, data); + else + { + interval2itm(*span, itm); + EncodeInterval(itm, IntervalStyle, data); + /* + * XXX: making EncodeInterval returns a string len is error-prone for me. + * so call strlen directly on the result. + */ + data_len = strlen(data); + } + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + /* * interval_recv - converts external binary format to interval */ @@ -1577,26 +1663,40 @@ out_of_range: /* EncodeSpecialTimestamp() * Convert reserved timestamp data type to string. */ -void +int EncodeSpecialTimestamp(Timestamp dt, char *str) { if (TIMESTAMP_IS_NOBEGIN(dt)) + { strcpy(str, EARLY); + return strlen(EARLY); + } else if (TIMESTAMP_IS_NOEND(dt)) + { strcpy(str, LATE); + return strlen(LATE); + } else /* shouldn't happen */ elog(ERROR, "invalid argument for EncodeSpecialTimestamp"); } -static void +static int EncodeSpecialInterval(const Interval *interval, char *str) { if (INTERVAL_IS_NOBEGIN(interval)) + { strcpy(str, EARLY); + return strlen(EARLY); + } else if (INTERVAL_IS_NOEND(interval)) + { strcpy(str, LATE); + return strlen(LATE); + } else /* shouldn't happen */ elog(ERROR, "invalid argument for EncodeSpecialInterval"); + + return 0; } Datum diff --git a/src/backend/utils/adt/varchar.c b/src/backend/utils/adt/varchar.c index a62e55eec19..e14c66999a8 100644 --- a/src/backend/utils/adt/varchar.c +++ b/src/backend/utils/adt/varchar.c @@ -223,6 +223,25 @@ bpcharout(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(TextDatumGetCString(txt)); } +Datum +bpcharprint(PG_FUNCTION_ARGS) +{ + Datum txt = PG_GETARG_DATUM(0); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + + /* XXX: improve here since we can put the cstring into buf directly. */ + char *data = TextDatumGetCString(txt); + uint32 data_len = strlen(data); + char *target; + + target = outStringReserveLen(buf, data_len); + memcpy(target, data, data_len); + outStringCompletePhase(buf, data_len); + + PG_RETURN_VOID(); +} + + /* * bpcharrecv - converts external binary format to bpchar */ @@ -520,6 +539,12 @@ varcharout(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(TextDatumGetCString(txt)); } +Datum +varcharprint(PG_FUNCTION_ARGS) +{ + return bpcharprint(fcinfo); +} + /* * varcharrecv - converts external binary format to varchar */ diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index c0ff51bd2fc..3fa6bff1182 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -293,6 +293,22 @@ textout(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(TextDatumGetCString(txt)); } + +Datum +textprint(PG_FUNCTION_ARGS) +{ + text *txt = (text *) pg_detoast_datum((struct varlena *)PG_GETARG_POINTER(0)); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(1); + uint32 text_len = VARSIZE(txt) - VARHDRSZ; + char *data; + + data = outStringReserveLen(buf, text_len); + memcpy(data, VARDATA(txt), text_len); + outStringCompletePhase(buf, text_len); + + PG_RETURN_VOID(); +} + /* * textrecv - converts external binary format to text */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fa9ae79082b..938f5e2d585 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -4877,7 +4877,6 @@ { oid => '1799', descr => 'I/O', proname => 'oidout', prorettype => 'cstring', proargtypes => 'oid', prosrc => 'oidout' }, - { oid => '3058', descr => 'concatenate values', proname => 'concat', provariadic => 'any', proisstrict => 'f', provolatile => 's', prorettype => 'text', proargtypes => 'any', @@ -12769,4 +12768,86 @@ proname => 'hashoid8extended', prorettype => 'int8', proargtypes => 'oid8 int8', prosrc => 'hashoid8extended' }, +{ + oid => '9771', descr => 'I/O', + proname => 'oidprint', prorettype => 'void', proargtypes => 'oid internal', + prosrc => 'oidprint'}, +{ + oid => '8907', descr => 'I/O', + proname => 'textprint', prorettype => 'void', proargtypes => 'text internal', + prosrc => 'textprint' }, + +{ + oid => '9234', descr => 'I/O', + proname => 'float4print', prorettype => 'void', proargtypes => 'float4 internal', + prosrc => 'float4print' }, +{ + oid => '6313', descr => 'I/O', + proname => 'float8print', prorettype => 'void', proargtypes => 'float8 internal', + prosrc => 'float8print' }, + +{ + oid => '4099', descr => 'I/O', + proname => 'int2print', prorettype => 'void', proargtypes => 'int2 internal', + prosrc => 'int2print' }, +{ + oid => '4100', descr => 'I/O', + proname => 'int4print', prorettype => 'void', proargtypes => 'int4 internal', + prosrc => 'int4print' }, +{ + oid => '4551', descr => 'I/O', + proname => 'int8print', prorettype => 'void', proargtypes => 'int8 internal', + prosrc => 'int8print' }, + +{ + oid => '4552', descr => 'I/O', + proname => 'timeprint', prorettype => 'void', proargtypes => 'time internal', + prosrc => 'time_print' }, + +{ + oid => '4553', descr => 'I/O', + proname => 'timetzprint', prorettype => 'void', proargtypes => 'timetz internal', + prosrc => 'timetz_print' }, + +{ + oid => '4554', descr => 'I/O', + proname => 'dateprint', prorettype => 'void', proargtypes => 'date internal', + prosrc => 'date_print'}, + + +{ + oid => '4555', descr => 'I/O', + proname => 'timestampprint', prorettype => 'void', proargtypes => 'timestamp internal', + prosrc => 'timestamp_print'}, + +{ + oid => '4556', descr => 'I/O', + proname => 'timestamptzprint', prorettype => 'void', proargtypes => 'timestamptz internal', + prosrc => 'timestamptz_print'}, + +{ + oid => '4557', descr => 'I/O', + proname => 'interval_print', prorettype => 'void', proargtypes => 'interval internal', + prosrc => 'interval_print'}, + +{ + oid => '4558', descr => 'I/O', + proname => 'numeric_print', prorettype => 'void', proargtypes => 'numeric internal', + prosrc => 'numeric_print'}, + +{ + oid => '4559', descr => 'I/O', + proname => 'charprint', prorettype => 'void', proargtypes => 'char internal', + prosrc => 'charprint'}, + +{ + oid => '4560', descr => 'I/O', + proname => 'bpcharprint', prorettype => 'void', proargtypes => 'bpchar internal', + prosrc => 'bpcharprint'}, + +{ + oid => '4561', descr => 'I/O', + proname => 'varcharprint', prorettype => 'void', proargtypes => 'varchar internal', + prosrc => 'varcharprint'}, + ] diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h index 079652c8ce4..6318fe8be5b 100644 --- a/src/include/lib/stringinfo.h +++ b/src/include/lib/stringinfo.h @@ -267,4 +267,23 @@ extern void enlargeStringInfo(StringInfo str, int needed); */ extern void destroyStringInfo(StringInfo str); +/* + * outString - The StringInfo used in type specific out function. + */ +static inline char * +outStringReserveLen(StringInfo buf, uint32 data_len) +{ + /* sizeof(uint32) is for storing the data_len itself. */ + enlargeStringInfo(buf, sizeof(uint32) + data_len); + return buf->data + buf->len + sizeof(uint32); +} + +/* define outStringCompletePhase as macro to avoid including pg_bswap.h */ +#define outStringCompletePhase(buf, data_len) \ +{ \ + *(uint32 *)(buf->data + buf->len) = pg_hton32(data_len); \ + buf->len += sizeof(uint32) + data_len; \ +} + + #endif /* STRINGINFO_H */ diff --git a/src/include/utils/date.h b/src/include/utils/date.h index 6063810891e..3a2fe34061e 100644 --- a/src/include/utils/date.h +++ b/src/include/utils/date.h @@ -111,7 +111,7 @@ extern DateADT timestamptz2date_safe(TimestampTz timestamp, Node *escontext); extern int32 date_cmp_timestamp_internal(DateADT dateVal, Timestamp dt2); extern int32 date_cmp_timestamptz_internal(DateADT dateVal, TimestampTz dt2); -extern void EncodeSpecialDate(DateADT dt, char *str); +extern int EncodeSpecialDate(DateADT dt, char *str); extern DateADT GetSQLCurrentDate(void); extern TimeTzADT *GetSQLCurrentTime(int32 typmod); extern TimeADT GetSQLLocalTime(int32 typmod); diff --git a/src/include/utils/datetime.h b/src/include/utils/datetime.h index f77c6acd8b6..a6754195e0b 100644 --- a/src/include/utils/datetime.h +++ b/src/include/utils/datetime.h @@ -330,11 +330,11 @@ extern int DetermineTimeZoneAbbrevOffset(struct pg_tm *tm, const char *abbr, pg_ extern int DetermineTimeZoneAbbrevOffsetTS(TimestampTz ts, const char *abbr, pg_tz *tzp, int *isdst); -extern void EncodeDateOnly(struct pg_tm *tm, int style, char *str); -extern void EncodeTimeOnly(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, int style, char *str); -extern void EncodeDateTime(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, const char *tzn, int style, char *str); +extern int EncodeDateOnly(struct pg_tm *tm, int style, char *str); +extern int EncodeTimeOnly(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, int style, char *str); +extern int EncodeDateTime(struct pg_tm *tm, fsec_t fsec, bool print_tz, int tz, const char *tzn, int style, char *str); extern void EncodeInterval(struct pg_itm *itm, int style, char *str); -extern void EncodeSpecialTimestamp(Timestamp dt, char *str); +extern int EncodeSpecialTimestamp(Timestamp dt, char *str); extern int ValidateDate(int fmask, bool isjulian, bool is2digits, bool bc, struct pg_tm *tm); -- 2.43.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
end of thread, other threads:[~2026-05-23 14:09 UTC | newest] Thread overview: 330+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2024-09-12 10:03 [PATCH v20260912 4/4] Make printtup a bit faster (intermediate state). Andy Fan <zhihuifan1213@163.com> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox